From: Steven G. Johnson Date: Fri, 6 Mar 2015 22:36:08 +0000 (-0500) Subject: directory cleanup: move tests and data into subdirectories X-Git-Tag: archive/raspbian/2.11.3-2+rpi1^2~24^2^2~218 X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/%22/%22http:/www.example.com/cgi/%22?a=commitdiff_plain;h=90721f2d39b0cdd5d22409f1bf4f6ce4b7382944;p=utf8proc.git directory cleanup: move tests and data into subdirectories --- diff --git a/Makefile b/Makefile index 8517192..678f5b6 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ all: c-library c-library: libutf8proc.a libutf8proc.$(SHLIB_EXT) clean: - rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_EXT) normtest graphemetest UnicodeData.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt + rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_EXT) test/normtest test/graphemetest data/UnicodeData.txt data/DerivedCoreProperties.txt data/CompositionExclusions.txt data/CaseFolding.txt data/NormalizationTest.txt data/GraphemeBreakTest.txt $(MAKE) -C bench clean update: utf8proc_data.c.new @@ -33,23 +33,23 @@ update: utf8proc_data.c.new # real targets -utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt - $(RUBY) data_generator.rb < UnicodeData.txt > utf8proc_data.c.new +utf8proc_data.c.new: data/data_generator.rb data/UnicodeData.txt data/GraphemeBreakProperty.txt data/DerivedCoreProperties.txt data/CompositionExclusions.txt data/CaseFolding.txt + (cd data; $(RUBY) data_generator.rb < UnicodeData.txt) > utf8proc_data.c.new -UnicodeData.txt: - $(CURL) -O http://www.unicode.org/Public/UNIDATA/UnicodeData.txt +data/UnicodeData.txt: + $(CURL) -o $@ -O http://www.unicode.org/Public/UNIDATA/UnicodeData.txt -GraphemeBreakProperty.txt: - $(CURL) -O http://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt +data/GraphemeBreakProperty.txt: + $(CURL) -o $@ -O http://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt -DerivedCoreProperties.txt: - $(CURL) -O http://www.unicode.org/Public/UNIDATA/DerivedCoreProperties.txt +data/DerivedCoreProperties.txt: + $(CURL) -o $@ -O http://www.unicode.org/Public/UNIDATA/DerivedCoreProperties.txt -CompositionExclusions.txt: - $(CURL) -O http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt +data/CompositionExclusions.txt: + $(CURL) -o $@ -O http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt -CaseFolding.txt: - $(CURL) -O http://www.unicode.org/Public/UNIDATA/CaseFolding.txt +data/CaseFolding.txt: + $(CURL) -o $@ -O http://www.unicode.org/Public/UNIDATA/CaseFolding.txt utf8proc.o: utf8proc.h utf8proc.c utf8proc_data.c $(cc) -c -o utf8proc.o utf8proc.c @@ -68,21 +68,21 @@ libutf8proc.dylib: utf8proc.o # Test programs -NormalizationTest.txt: - $(CURL) -O http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt +data/NormalizationTest.txt: + $(CURL) -o $@ -O http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt -GraphemeBreakTest.txt: +data/GraphemeBreakTest.txt: $(CURL) http://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@ -normtest: normtest.c utf8proc.o utf8proc.h tests.h - $(cc) normtest.c utf8proc.o -o $@ +test/normtest: test/normtest.c utf8proc.o utf8proc.h test/tests.h + $(cc) test/normtest.c utf8proc.o -o $@ -graphemetest: graphemetest.c utf8proc.o utf8proc.h tests.h - $(cc) graphemetest.c utf8proc.o -o $@ +test/graphemetest: test/graphemetest.c utf8proc.o utf8proc.h test/tests.h + $(cc) test/graphemetest.c utf8proc.o -o $@ -printproperty: printproperty.c utf8proc.o utf8proc.h tests.h - $(cc) printproperty.c utf8proc.o -o $@ +test/printproperty: test/printproperty.c utf8proc.o utf8proc.h test/tests.h + $(cc) test/printproperty.c utf8proc.o -o $@ -check: normtest NormalizationTest.txt graphemetest GraphemeBreakTest.txt - ./normtest - ./graphemetest +check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt + test/normtest data/NormalizationTest.txt + test/graphemetest data/GraphemeBreakTest.txt diff --git a/data/data_generator.rb b/data/data_generator.rb new file mode 100644 index 0000000..1439038 --- /dev/null +++ b/data/data_generator.rb @@ -0,0 +1,317 @@ +#!/usr/bin/env ruby + +# This file was used to generate the 'unicode_data.c' file by parsing the +# Unicode data file 'UnicodeData.txt' of the Unicode Character Database. +# It is included for informational purposes only and not intended for +# production use. + + +# Copyright (c) 2009 Public Software Group e. V., Berlin, Germany +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + + +# This file contains derived data from a modified version of the +# Unicode data files. The following license applies to that data: +# +# COPYRIGHT AND PERMISSION NOTICE +# +# Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed +# under the Terms of Use in http://www.unicode.org/copyright.html. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of the Unicode data files and any associated documentation (the "Data +# Files") or Unicode software and any associated documentation (the +# "Software") to deal in the Data Files or Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, and/or sell copies of the Data Files or Software, and +# to permit persons to whom the Data Files or Software are furnished to do +# so, provided that (a) the above copyright notice(s) and this permission +# notice appear with all copies of the Data Files or Software, (b) both the +# above copyright notice(s) and this permission notice appear in associated +# documentation, and (c) there is clear notice in each modified Data File or +# in the Software as well as in the documentation associated with the Data +# File(s) or Software that the data or software has been modified. +# +# THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +# THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS +# INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR +# CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THE DATA FILES OR SOFTWARE. +# +# Except as contained in this notice, the name of a copyright holder shall +# not be used in advertising or otherwise to promote the sale, use or other +# dealings in these Data Files or Software without prior written +# authorization of the copyright holder. + + +$ignorable_list = File.read("DerivedCoreProperties.txt")[/# Derived Property: Default_Ignorable_Code_Point.*?# Total code points:/m] +$ignorable = [] +$ignorable_list.each_line do |entry| + if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)/ + $1.hex.upto($2.hex) { |e2| $ignorable << e2 } + elsif entry =~ /^[0-9A-F]+/ + $ignorable << $&.hex + end +end + +$grapheme_boundclass_list = File.read("GraphemeBreakProperty.txt") +$grapheme_boundclass = Hash.new("UTF8PROC_BOUNDCLASS_OTHER") +$grapheme_boundclass_list.each_line do |entry| + if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([A-Za-z_]+)/ + $1.hex.upto($2.hex) { |e2| $grapheme_boundclass[e2] = "UTF8PROC_BOUNDCLASS_" + $3.upcase } + elsif entry =~ /^([0-9A-F]+)\s*;\s*([A-Za-z_]+)/ + $grapheme_boundclass[$1.hex] = "UTF8PROC_BOUNDCLASS_" + $2.upcase + end +end + +$exclusions = File.read("CompositionExclusions.txt")[/# \(1\) Script Specifics.*?# Total code points:/m] +$exclusions = $exclusions.chomp.split("\n").collect { |e| e.hex } + +$excl_version = File.read("CompositionExclusions.txt")[/# \(2\) Post Composition Version precomposed characters.*?# Total code points:/m] +$excl_version = $excl_version.chomp.split("\n").collect { |e| e.hex } + +$case_folding_string = File.open("CaseFolding.txt").read + +$case_folding = {} +$case_folding_string.chomp.split("\n").each do |line| + next unless line =~ /([0-9A-F]+); [CFS]; ([0-9A-F ]+);/i + $case_folding[$1.hex] = $2.split(" ").collect { |e| e.hex } +end + +$int_array = [] +$int_array_indicies = {} + +def str2c(string, prefix) + return "0" if string.nil? + return "UTF8PROC_#{prefix}_#{string.upcase}" +end +def ary2c(array) + return "NULL" if array.nil? + unless $int_array_indicies[array] + $int_array_indicies[array] = $int_array.length + array.each { |entry| $int_array << entry } + $int_array << -1 + end + return "utf8proc_sequences + #{$int_array_indicies[array]}" +end + +class UnicodeChar + attr_accessor :code, :name, :category, :combining_class, :bidi_class, + :decomp_type, :decomp_mapping, + :bidi_mirrored, + :uppercase_mapping, :lowercase_mapping, :titlecase_mapping + def initialize(line) + raise "Could not parse input." unless line =~ /^ + ([0-9A-F]+); # code + ([^;]+); # name + ([A-Z]+); # general category + ([0-9]+); # canonical combining class + ([A-Z]+); # bidi class + (<([A-Z]*)>)? # decomposition type + ((\ ?[0-9A-F]+)*); # decompomposition mapping + ([0-9]*); # decimal digit + ([0-9]*); # digit + ([^;]*); # numeric + ([YN]*); # bidi mirrored + ([^;]*); # unicode 1.0 name + ([^;]*); # iso comment + ([0-9A-F]*); # simple uppercase mapping + ([0-9A-F]*); # simple lowercase mapping + ([0-9A-F]*)$/ix # simple titlecase mapping + @code = $1.hex + @name = $2 + @category = $3 + @combining_class = Integer($4) + @bidi_class = $5 + @decomp_type = $7 + @decomp_mapping = ($8=='') ? nil : + $8.split.collect { |element| element.hex } + @bidi_mirrored = ($13=='Y') ? true : false + @uppercase_mapping = ($16=='') ? nil : $16.hex + @lowercase_mapping = ($17=='') ? nil : $17.hex + @titlecase_mapping = ($18=='') ? nil : $18.hex + end + def case_folding + $case_folding[code] + end + def c_entry(comb1_indicies, comb2_indicies) + " " << + "{#{str2c category, 'CATEGORY'}, #{combining_class}, " << + "#{str2c bidi_class, 'BIDI_CLASS'}, " << + "#{str2c decomp_type, 'DECOMP_TYPE'}, " << + "#{ary2c decomp_mapping}, " << + "#{ary2c case_folding}, " << + "#{uppercase_mapping or -1}, " << + "#{lowercase_mapping or -1}, " << + "#{titlecase_mapping or -1}, " << + "#{comb1_indicies[code] ? + (comb1_indicies[code]*comb2_indicies.keys.length) : -1 + }, #{comb2_indicies[code] or -1}, " << + "#{bidi_mirrored}, " << + "#{$exclusions.include?(code) or $excl_version.include?(code)}, " << + "#{$ignorable.include?(code)}, " << + "#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " << + "#{$grapheme_boundclass[code]}},\n" + end +end + +chars = [] +char_hash = {} + +while gets + if $_ =~ /^([0-9A-F]+);<[^;>,]+, First>;/i + first = $1.hex + gets + char = UnicodeChar.new($_) + raise "No last character of sequence found." unless + $_ =~ /^([0-9A-F]+);<([^;>,]+), Last>;/i + last = $1.hex + name = "<#{$2}>" + for i in first..last + char_clone = char.clone + char_clone.code = i + char_clone.name = name + char_hash[char_clone.code] = char_clone + chars << char_clone + end + else + char = UnicodeChar.new($_) + char_hash[char.code] = char + chars << char + end +end + +comb1st_indicies = {} +comb2nd_indicies = {} +comb_array = [] + +chars.each do |char| + if !char.nil? and char.decomp_type.nil? and char.decomp_mapping and + char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and + char_hash[char.decomp_mapping[0]].combining_class == 0 and + not $exclusions.include?(char.code) + unless comb1st_indicies[char.decomp_mapping[0]] + comb1st_indicies[char.decomp_mapping[0]] = comb1st_indicies.keys.length + end + unless comb2nd_indicies[char.decomp_mapping[1]] + comb2nd_indicies[char.decomp_mapping[1]] = comb2nd_indicies.keys.length + end + comb_array[comb1st_indicies[char.decomp_mapping[0]]] ||= [] + raise "Duplicate canonical mapping" if + comb_array[comb1st_indicies[char.decomp_mapping[0]]][ + comb2nd_indicies[char.decomp_mapping[1]]] + comb_array[comb1st_indicies[char.decomp_mapping[0]]][ + comb2nd_indicies[char.decomp_mapping[1]]] = char.code + end +end + +properties_indicies = {} +properties = [] +chars.each do |char| + c_entry = char.c_entry(comb1st_indicies, comb2nd_indicies) + unless properties_indicies[c_entry] + properties_indicies[c_entry] = properties.length + properties << c_entry + end +end + +stage1 = [] +stage2 = [] +for code in 0...0x110000 + next unless code % 0x100 == 0 + stage2_entry = [] + for code2 in code...(code+0x100) + if char_hash[code2] + stage2_entry << (properties_indicies[char_hash[code2].c_entry( + comb1st_indicies, comb2nd_indicies)] + 1) + else + stage2_entry << 0 + end + end + old_index = stage2.index(stage2_entry) + if old_index + stage1 << (old_index * 0x100) + else + stage1 << (stage2.length * 0x100) + stage2 << stage2_entry + end +end + +$stdout << "const int32_t utf8proc_sequences[] = {\n " +i = 0 +$int_array.each do |entry| + i += 1 + if i == 8 + i = 0 + $stdout << "\n " + end + $stdout << entry << ", " +end +$stdout << "};\n\n" + +$stdout << "const uint16_t utf8proc_stage1table[] = {\n " +i = 0 +stage1.each do |entry| + i += 1 + if i == 8 + i = 0 + $stdout << "\n " + end + $stdout << entry << ", " +end +$stdout << "};\n\n" + +$stdout << "const uint16_t utf8proc_stage2table[] = {\n " +i = 0 +stage2.flatten.each do |entry| + i += 1 + if i == 8 + i = 0 + $stdout << "\n " + end + $stdout << entry << ", " +end +$stdout << "};\n\n" + +$stdout << "const utf8proc_property_t utf8proc_properties[] = {\n" +$stdout << " {0, 0, 0, 0, NULL, NULL, -1, -1, -1, -1, -1, false,false,false,false, UTF8PROC_BOUNDCLASS_OTHER},\n" +properties.each { |line| + $stdout << line +} +$stdout << "};\n\n" + +$stdout << "const int32_t utf8proc_combinations[] = {\n " +i = 0 +comb1st_indicies.keys.each_index do |a| + comb2nd_indicies.keys.each_index do |b| + i += 1 + if i == 8 + i = 0 + $stdout << "\n " + end + $stdout << ( comb_array[a][b] or -1 ) << ", " + end +end +$stdout << "};\n\n" + diff --git a/data_generator.rb b/data_generator.rb deleted file mode 100644 index 1439038..0000000 --- a/data_generator.rb +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env ruby - -# This file was used to generate the 'unicode_data.c' file by parsing the -# Unicode data file 'UnicodeData.txt' of the Unicode Character Database. -# It is included for informational purposes only and not intended for -# production use. - - -# Copyright (c) 2009 Public Software Group e. V., Berlin, Germany -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - - -# This file contains derived data from a modified version of the -# Unicode data files. The following license applies to that data: -# -# COPYRIGHT AND PERMISSION NOTICE -# -# Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed -# under the Terms of Use in http://www.unicode.org/copyright.html. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of the Unicode data files and any associated documentation (the "Data -# Files") or Unicode software and any associated documentation (the -# "Software") to deal in the Data Files or Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, and/or sell copies of the Data Files or Software, and -# to permit persons to whom the Data Files or Software are furnished to do -# so, provided that (a) the above copyright notice(s) and this permission -# notice appear with all copies of the Data Files or Software, (b) both the -# above copyright notice(s) and this permission notice appear in associated -# documentation, and (c) there is clear notice in each modified Data File or -# in the Software as well as in the documentation associated with the Data -# File(s) or Software that the data or software has been modified. -# -# THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF -# THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS -# INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR -# CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF -# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -# PERFORMANCE OF THE DATA FILES OR SOFTWARE. -# -# Except as contained in this notice, the name of a copyright holder shall -# not be used in advertising or otherwise to promote the sale, use or other -# dealings in these Data Files or Software without prior written -# authorization of the copyright holder. - - -$ignorable_list = File.read("DerivedCoreProperties.txt")[/# Derived Property: Default_Ignorable_Code_Point.*?# Total code points:/m] -$ignorable = [] -$ignorable_list.each_line do |entry| - if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)/ - $1.hex.upto($2.hex) { |e2| $ignorable << e2 } - elsif entry =~ /^[0-9A-F]+/ - $ignorable << $&.hex - end -end - -$grapheme_boundclass_list = File.read("GraphemeBreakProperty.txt") -$grapheme_boundclass = Hash.new("UTF8PROC_BOUNDCLASS_OTHER") -$grapheme_boundclass_list.each_line do |entry| - if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([A-Za-z_]+)/ - $1.hex.upto($2.hex) { |e2| $grapheme_boundclass[e2] = "UTF8PROC_BOUNDCLASS_" + $3.upcase } - elsif entry =~ /^([0-9A-F]+)\s*;\s*([A-Za-z_]+)/ - $grapheme_boundclass[$1.hex] = "UTF8PROC_BOUNDCLASS_" + $2.upcase - end -end - -$exclusions = File.read("CompositionExclusions.txt")[/# \(1\) Script Specifics.*?# Total code points:/m] -$exclusions = $exclusions.chomp.split("\n").collect { |e| e.hex } - -$excl_version = File.read("CompositionExclusions.txt")[/# \(2\) Post Composition Version precomposed characters.*?# Total code points:/m] -$excl_version = $excl_version.chomp.split("\n").collect { |e| e.hex } - -$case_folding_string = File.open("CaseFolding.txt").read - -$case_folding = {} -$case_folding_string.chomp.split("\n").each do |line| - next unless line =~ /([0-9A-F]+); [CFS]; ([0-9A-F ]+);/i - $case_folding[$1.hex] = $2.split(" ").collect { |e| e.hex } -end - -$int_array = [] -$int_array_indicies = {} - -def str2c(string, prefix) - return "0" if string.nil? - return "UTF8PROC_#{prefix}_#{string.upcase}" -end -def ary2c(array) - return "NULL" if array.nil? - unless $int_array_indicies[array] - $int_array_indicies[array] = $int_array.length - array.each { |entry| $int_array << entry } - $int_array << -1 - end - return "utf8proc_sequences + #{$int_array_indicies[array]}" -end - -class UnicodeChar - attr_accessor :code, :name, :category, :combining_class, :bidi_class, - :decomp_type, :decomp_mapping, - :bidi_mirrored, - :uppercase_mapping, :lowercase_mapping, :titlecase_mapping - def initialize(line) - raise "Could not parse input." unless line =~ /^ - ([0-9A-F]+); # code - ([^;]+); # name - ([A-Z]+); # general category - ([0-9]+); # canonical combining class - ([A-Z]+); # bidi class - (<([A-Z]*)>)? # decomposition type - ((\ ?[0-9A-F]+)*); # decompomposition mapping - ([0-9]*); # decimal digit - ([0-9]*); # digit - ([^;]*); # numeric - ([YN]*); # bidi mirrored - ([^;]*); # unicode 1.0 name - ([^;]*); # iso comment - ([0-9A-F]*); # simple uppercase mapping - ([0-9A-F]*); # simple lowercase mapping - ([0-9A-F]*)$/ix # simple titlecase mapping - @code = $1.hex - @name = $2 - @category = $3 - @combining_class = Integer($4) - @bidi_class = $5 - @decomp_type = $7 - @decomp_mapping = ($8=='') ? nil : - $8.split.collect { |element| element.hex } - @bidi_mirrored = ($13=='Y') ? true : false - @uppercase_mapping = ($16=='') ? nil : $16.hex - @lowercase_mapping = ($17=='') ? nil : $17.hex - @titlecase_mapping = ($18=='') ? nil : $18.hex - end - def case_folding - $case_folding[code] - end - def c_entry(comb1_indicies, comb2_indicies) - " " << - "{#{str2c category, 'CATEGORY'}, #{combining_class}, " << - "#{str2c bidi_class, 'BIDI_CLASS'}, " << - "#{str2c decomp_type, 'DECOMP_TYPE'}, " << - "#{ary2c decomp_mapping}, " << - "#{ary2c case_folding}, " << - "#{uppercase_mapping or -1}, " << - "#{lowercase_mapping or -1}, " << - "#{titlecase_mapping or -1}, " << - "#{comb1_indicies[code] ? - (comb1_indicies[code]*comb2_indicies.keys.length) : -1 - }, #{comb2_indicies[code] or -1}, " << - "#{bidi_mirrored}, " << - "#{$exclusions.include?(code) or $excl_version.include?(code)}, " << - "#{$ignorable.include?(code)}, " << - "#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " << - "#{$grapheme_boundclass[code]}},\n" - end -end - -chars = [] -char_hash = {} - -while gets - if $_ =~ /^([0-9A-F]+);<[^;>,]+, First>;/i - first = $1.hex - gets - char = UnicodeChar.new($_) - raise "No last character of sequence found." unless - $_ =~ /^([0-9A-F]+);<([^;>,]+), Last>;/i - last = $1.hex - name = "<#{$2}>" - for i in first..last - char_clone = char.clone - char_clone.code = i - char_clone.name = name - char_hash[char_clone.code] = char_clone - chars << char_clone - end - else - char = UnicodeChar.new($_) - char_hash[char.code] = char - chars << char - end -end - -comb1st_indicies = {} -comb2nd_indicies = {} -comb_array = [] - -chars.each do |char| - if !char.nil? and char.decomp_type.nil? and char.decomp_mapping and - char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and - char_hash[char.decomp_mapping[0]].combining_class == 0 and - not $exclusions.include?(char.code) - unless comb1st_indicies[char.decomp_mapping[0]] - comb1st_indicies[char.decomp_mapping[0]] = comb1st_indicies.keys.length - end - unless comb2nd_indicies[char.decomp_mapping[1]] - comb2nd_indicies[char.decomp_mapping[1]] = comb2nd_indicies.keys.length - end - comb_array[comb1st_indicies[char.decomp_mapping[0]]] ||= [] - raise "Duplicate canonical mapping" if - comb_array[comb1st_indicies[char.decomp_mapping[0]]][ - comb2nd_indicies[char.decomp_mapping[1]]] - comb_array[comb1st_indicies[char.decomp_mapping[0]]][ - comb2nd_indicies[char.decomp_mapping[1]]] = char.code - end -end - -properties_indicies = {} -properties = [] -chars.each do |char| - c_entry = char.c_entry(comb1st_indicies, comb2nd_indicies) - unless properties_indicies[c_entry] - properties_indicies[c_entry] = properties.length - properties << c_entry - end -end - -stage1 = [] -stage2 = [] -for code in 0...0x110000 - next unless code % 0x100 == 0 - stage2_entry = [] - for code2 in code...(code+0x100) - if char_hash[code2] - stage2_entry << (properties_indicies[char_hash[code2].c_entry( - comb1st_indicies, comb2nd_indicies)] + 1) - else - stage2_entry << 0 - end - end - old_index = stage2.index(stage2_entry) - if old_index - stage1 << (old_index * 0x100) - else - stage1 << (stage2.length * 0x100) - stage2 << stage2_entry - end -end - -$stdout << "const int32_t utf8proc_sequences[] = {\n " -i = 0 -$int_array.each do |entry| - i += 1 - if i == 8 - i = 0 - $stdout << "\n " - end - $stdout << entry << ", " -end -$stdout << "};\n\n" - -$stdout << "const uint16_t utf8proc_stage1table[] = {\n " -i = 0 -stage1.each do |entry| - i += 1 - if i == 8 - i = 0 - $stdout << "\n " - end - $stdout << entry << ", " -end -$stdout << "};\n\n" - -$stdout << "const uint16_t utf8proc_stage2table[] = {\n " -i = 0 -stage2.flatten.each do |entry| - i += 1 - if i == 8 - i = 0 - $stdout << "\n " - end - $stdout << entry << ", " -end -$stdout << "};\n\n" - -$stdout << "const utf8proc_property_t utf8proc_properties[] = {\n" -$stdout << " {0, 0, 0, 0, NULL, NULL, -1, -1, -1, -1, -1, false,false,false,false, UTF8PROC_BOUNDCLASS_OTHER},\n" -properties.each { |line| - $stdout << line -} -$stdout << "};\n\n" - -$stdout << "const int32_t utf8proc_combinations[] = {\n " -i = 0 -comb1st_indicies.keys.each_index do |a| - comb2nd_indicies.keys.each_index do |b| - i += 1 - if i == 8 - i = 0 - $stdout << "\n " - end - $stdout << ( comb_array[a][b] or -1 ) << ", " - end -end -$stdout << "};\n\n" - diff --git a/graphemetest.c b/graphemetest.c deleted file mode 100644 index 094fa7f..0000000 --- a/graphemetest.c +++ /dev/null @@ -1,72 +0,0 @@ -#include "tests.h" - -int main(void) -{ - char *buf = NULL; - size_t bufsize = 0; - FILE *f = fopen("GraphemeBreakTest.txt", "r"); - uint8_t src[1024]; - - check(f != NULL, "error opening GraphemeBreakTest.txt"); - while (getline(&buf, &bufsize, f) > 0) { - size_t bi = 0, si = 0; - lineno += 1; - - if (lineno % 100 == 0) - printf("checking line %zd...\n", lineno); - - if (buf[0] == '#') continue; - - while (buf[bi]) { - bi = skipspaces(buf, bi); - if (buf[bi] == '/') { /* grapheme break */ - src[si++] = '/'; - bi++; - } - else if (buf[bi] == '+') { /* no break */ - bi++; - } - else if (buf[bi] == '#') { /* start of comments */ - break; - } - else { /* hex-encoded codepoint */ - bi += encode((char*) (src + si), buf + bi) - 1; - while (src[si]) ++si; /* advance to NUL termination */ - } - } - if (si && src[si-1] == '/') - --si; /* no break after final grapheme */ - src[si] = 0; /* NUL-terminate */ - - if (si) { - uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ - size_t i = 0, j = 0; - ssize_t glen; - uint8_t *g; /* utf8proc_map grapheme results */ - while (i < si) { - if (src[i] != '/') - utf8[j++] = src[i++]; - else - i++; - } - glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND); - if (glen == UTF8PROC_ERROR_INVALIDUTF8) { - /* the test file contains surrogate codepoints, which are only for UTF-16 */ - printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); - } - else { - check(glen >= 0, "utf8proc_map error = %s", - utf8proc_errmsg(glen)); - for (i = 0; i <= glen; ++i) - if (g[i] == 0xff) - g[i] = '/'; /* easier-to-read output (/ is not in test strings) */ - check(!strcmp((char*)g, (char*)src), - "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); - } - free(g); - } - } - fclose(f); - printf("Passed tests after %zd lines!\n", lineno); - return 0; -} diff --git a/normtest.c b/normtest.c deleted file mode 100644 index 6ccd813..0000000 --- a/normtest.c +++ /dev/null @@ -1,64 +0,0 @@ -#include "tests.h" - -#define CHECK_NORM(NRM, norm, src) { \ - char *src_norm = (char*) utf8proc_ ## NRM((uint8_t*) src); \ - check(!strcmp(norm, src_norm), \ - "normalization failed for %s -> %s", src, norm); \ - free(src_norm); \ -} - -int main(void) -{ - char *buf = NULL; - size_t bufsize = 0; - FILE *f = fopen("NormalizationTest.txt", "r"); - char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024]; - - check(f != NULL, "error opening NormalizationTest.txt"); - while (getline(&buf, &bufsize, f) > 0) { - size_t offset; - lineno += 1; - - if (buf[0] == '@') { - printf("line %zd: %s", lineno, buf + 1); - continue; - } - else if (lineno % 1000 == 0) - printf("checking line %zd...\n", lineno); - - if (buf[0] == '#') continue; - - offset = encode(source, buf); - offset += encode(NFC, buf + offset); - offset += encode(NFD, buf + offset); - offset += encode(NFKC, buf + offset); - offset += encode(NFKD, buf + offset); - - CHECK_NORM(NFC, NFC, source); - CHECK_NORM(NFC, NFC, NFC); - CHECK_NORM(NFC, NFC, NFD); - CHECK_NORM(NFC, NFKC, NFKC); - CHECK_NORM(NFC, NFKC, NFKD); - - CHECK_NORM(NFD, NFD, source); - CHECK_NORM(NFD, NFD, NFC); - CHECK_NORM(NFD, NFD, NFD); - CHECK_NORM(NFD, NFKD, NFKC); - CHECK_NORM(NFD, NFKD, NFKD); - - CHECK_NORM(NFKC, NFKC, source); - CHECK_NORM(NFKC, NFKC, NFC); - CHECK_NORM(NFKC, NFKC, NFD); - CHECK_NORM(NFKC, NFKC, NFKC); - CHECK_NORM(NFKC, NFKC, NFKD); - - CHECK_NORM(NFKD, NFKD, source); - CHECK_NORM(NFKD, NFKD, NFC); - CHECK_NORM(NFKD, NFKD, NFD); - CHECK_NORM(NFKD, NFKD, NFKC); - CHECK_NORM(NFKD, NFKD, NFKD); - } - fclose(f); - printf("Passed tests after %zd lines!\n", lineno); - return 0; -} diff --git a/printproperty.c b/printproperty.c deleted file mode 100644 index 37e4fa3..0000000 --- a/printproperty.c +++ /dev/null @@ -1,45 +0,0 @@ -/* simple test program to print out the utf8proc properties for a codepoint */ - -#include "tests.h" - -int main(int argc, char **argv) -{ - int i; - - for (i = 1; i < argc; ++i) { - int c; - check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]); - const utf8proc_property_t *p = utf8proc_get_property(c); - printf("U+%s:\n" - " category = %d\n" - " combining_class = %d\n" - " bidi_class = %d\n" - " decomp_type = %d\n" - " uppercase_mapping = %x\n" - " lowercase_mapping = %x\n" - " titlecase_mapping = %x\n" - " comb1st_index = %d\n" - " comb2nd_index = %d\n" - " bidi_mirrored = %d\n" - " comp_exclusion = %d\n" - " ignorable = %d\n" - " control_boundary = %d\n" - " boundclass = %d\n", - argv[i], - p->category, - p->combining_class, - p->bidi_class, - p->decomp_type, - p->uppercase_mapping, - p->lowercase_mapping, - p->titlecase_mapping, - p->comb1st_index, - p->comb2nd_index, - p->bidi_mirrored, - p->comp_exclusion, - p->ignorable, - p->control_boundary, - p->boundclass); - } - return 0; -} diff --git a/test/graphemetest.c b/test/graphemetest.c new file mode 100644 index 0000000..7914dc8 --- /dev/null +++ b/test/graphemetest.c @@ -0,0 +1,72 @@ +#include "tests.h" + +int main(int argc, char **argv) +{ + char *buf = NULL; + size_t bufsize = 0; + FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; + uint8_t src[1024]; + + check(f != NULL, "error opening GraphemeBreakTest.txt"); + while (getline(&buf, &bufsize, f) > 0) { + size_t bi = 0, si = 0; + lineno += 1; + + if (lineno % 100 == 0) + printf("checking line %zd...\n", lineno); + + if (buf[0] == '#') continue; + + while (buf[bi]) { + bi = skipspaces(buf, bi); + if (buf[bi] == '/') { /* grapheme break */ + src[si++] = '/'; + bi++; + } + else if (buf[bi] == '+') { /* no break */ + bi++; + } + else if (buf[bi] == '#') { /* start of comments */ + break; + } + else { /* hex-encoded codepoint */ + bi += encode((char*) (src + si), buf + bi) - 1; + while (src[si]) ++si; /* advance to NUL termination */ + } + } + if (si && src[si-1] == '/') + --si; /* no break after final grapheme */ + src[si] = 0; /* NUL-terminate */ + + if (si) { + uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */ + size_t i = 0, j = 0; + ssize_t glen; + uint8_t *g; /* utf8proc_map grapheme results */ + while (i < si) { + if (src[i] != '/') + utf8[j++] = src[i++]; + else + i++; + } + glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND); + if (glen == UTF8PROC_ERROR_INVALIDUTF8) { + /* the test file contains surrogate codepoints, which are only for UTF-16 */ + printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno); + } + else { + check(glen >= 0, "utf8proc_map error = %s", + utf8proc_errmsg(glen)); + for (i = 0; i <= glen; ++i) + if (g[i] == 0xff) + g[i] = '/'; /* easier-to-read output (/ is not in test strings) */ + check(!strcmp((char*)g, (char*)src), + "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src); + } + free(g); + } + } + fclose(f); + printf("Passed tests after %zd lines!\n", lineno); + return 0; +} diff --git a/test/normtest.c b/test/normtest.c new file mode 100644 index 0000000..7add4c6 --- /dev/null +++ b/test/normtest.c @@ -0,0 +1,64 @@ +#include "tests.h" + +#define CHECK_NORM(NRM, norm, src) { \ + char *src_norm = (char*) utf8proc_ ## NRM((uint8_t*) src); \ + check(!strcmp(norm, src_norm), \ + "normalization failed for %s -> %s", src, norm); \ + free(src_norm); \ +} + +int main(int argc, char **argv) +{ + char *buf = NULL; + size_t bufsize = 0; + FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL; + char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024]; + + check(f != NULL, "error opening NormalizationTest.txt"); + while (getline(&buf, &bufsize, f) > 0) { + size_t offset; + lineno += 1; + + if (buf[0] == '@') { + printf("line %zd: %s", lineno, buf + 1); + continue; + } + else if (lineno % 1000 == 0) + printf("checking line %zd...\n", lineno); + + if (buf[0] == '#') continue; + + offset = encode(source, buf); + offset += encode(NFC, buf + offset); + offset += encode(NFD, buf + offset); + offset += encode(NFKC, buf + offset); + offset += encode(NFKD, buf + offset); + + CHECK_NORM(NFC, NFC, source); + CHECK_NORM(NFC, NFC, NFC); + CHECK_NORM(NFC, NFC, NFD); + CHECK_NORM(NFC, NFKC, NFKC); + CHECK_NORM(NFC, NFKC, NFKD); + + CHECK_NORM(NFD, NFD, source); + CHECK_NORM(NFD, NFD, NFC); + CHECK_NORM(NFD, NFD, NFD); + CHECK_NORM(NFD, NFKD, NFKC); + CHECK_NORM(NFD, NFKD, NFKD); + + CHECK_NORM(NFKC, NFKC, source); + CHECK_NORM(NFKC, NFKC, NFC); + CHECK_NORM(NFKC, NFKC, NFD); + CHECK_NORM(NFKC, NFKC, NFKC); + CHECK_NORM(NFKC, NFKC, NFKD); + + CHECK_NORM(NFKD, NFKD, source); + CHECK_NORM(NFKD, NFKD, NFC); + CHECK_NORM(NFKD, NFKD, NFD); + CHECK_NORM(NFKD, NFKD, NFKC); + CHECK_NORM(NFKD, NFKD, NFKD); + } + fclose(f); + printf("Passed tests after %zd lines!\n", lineno); + return 0; +} diff --git a/test/printproperty.c b/test/printproperty.c new file mode 100644 index 0000000..37e4fa3 --- /dev/null +++ b/test/printproperty.c @@ -0,0 +1,45 @@ +/* simple test program to print out the utf8proc properties for a codepoint */ + +#include "tests.h" + +int main(int argc, char **argv) +{ + int i; + + for (i = 1; i < argc; ++i) { + int c; + check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]); + const utf8proc_property_t *p = utf8proc_get_property(c); + printf("U+%s:\n" + " category = %d\n" + " combining_class = %d\n" + " bidi_class = %d\n" + " decomp_type = %d\n" + " uppercase_mapping = %x\n" + " lowercase_mapping = %x\n" + " titlecase_mapping = %x\n" + " comb1st_index = %d\n" + " comb2nd_index = %d\n" + " bidi_mirrored = %d\n" + " comp_exclusion = %d\n" + " ignorable = %d\n" + " control_boundary = %d\n" + " boundclass = %d\n", + argv[i], + p->category, + p->combining_class, + p->bidi_class, + p->decomp_type, + p->uppercase_mapping, + p->lowercase_mapping, + p->titlecase_mapping, + p->comb1st_index, + p->comb2nd_index, + p->bidi_mirrored, + p->comp_exclusion, + p->ignorable, + p->control_boundary, + p->boundclass); + } + return 0; +} diff --git a/test/tests.h b/test/tests.h new file mode 100644 index 0000000..d4897f8 --- /dev/null +++ b/test/tests.h @@ -0,0 +1,53 @@ +/* Common functions and includes for our test programs. */ + +#include +#include +#include +#include +#include + +#include "../utf8proc.h" + +size_t lineno = 0; + +void check(int cond, const char *format, ...) +{ + if (!cond) { + va_list args; + fprintf(stderr, "line %zd: ", lineno); + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + fprintf(stderr, "\n"); + exit(1); + } +} + +size_t skipspaces(const char *buf, size_t i) +{ + while (isspace(buf[i])) ++i; + return i; +} + +/* if buf points to a sequence of codepoints encoded as hexadecimal strings, + separated by whitespace, and terminated by any character not in + [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string + in dest, returning the number of bytes read from buf */ +size_t encode(char *dest, const char *buf) +{ + size_t i = 0, j, d = 0; + do { + int c; + i = skipspaces(buf, i); + for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j) + ; /* find end of hex input */ + if (j == i) { /* no codepoint found */ + dest[d] = 0; /* NUL-terminate destination string */ + return i + 1; + } + check(sscanf(buf + i, "%x", &c) == 1, "invalid hex input %s", buf+i); + i = j; /* skip to char after hex input */ + d += utf8proc_encode_char(c, (uint8_t *) (dest + d)); + } while (1); +} + diff --git a/tests.h b/tests.h deleted file mode 100644 index aee3a57..0000000 --- a/tests.h +++ /dev/null @@ -1,53 +0,0 @@ -/* Common functions and includes for our test programs. */ - -#include -#include -#include -#include -#include - -#include "utf8proc.h" - -size_t lineno = 0; - -void check(int cond, const char *format, ...) -{ - if (!cond) { - va_list args; - fprintf(stderr, "line %zd: ", lineno); - va_start(args, format); - vfprintf(stderr, format, args); - va_end(args); - fprintf(stderr, "\n"); - exit(1); - } -} - -size_t skipspaces(const char *buf, size_t i) -{ - while (isspace(buf[i])) ++i; - return i; -} - -/* if buf points to a sequence of codepoints encoded as hexadecimal strings, - separated by whitespace, and terminated by any character not in - [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string - in dest, returning the number of bytes read from buf */ -size_t encode(char *dest, const char *buf) -{ - size_t i = 0, j, d = 0; - do { - int c; - i = skipspaces(buf, i); - for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j) - ; /* find end of hex input */ - if (j == i) { /* no codepoint found */ - dest[d] = 0; /* NUL-terminate destination string */ - return i + 1; - } - check(sscanf(buf + i, "%x", &c) == 1, "invalid hex input %s", buf+i); - i = j; /* skip to char after hex input */ - d += utf8proc_encode_char(c, (uint8_t *) (dest + d)); - } while (1); -} -